<!DOCTYPE html><html lang="zh-CN" data-theme="dark"><head><meta charset="UTF-8"><meta http-equiv="X-UA-Compatible" content="IE=edge"><meta name="viewport" content="width=device-width, initial-scale=1.0, maximum-scale=1.0, user-scalable=no"><title>Pod异常状态排错 | J.のblog</title><meta name="keywords" content="kubernetes,云原生"><meta name="author" content="J."><meta name="copyright" content="J."><meta name="format-detection" content="telephone=no"><meta name="theme-color" content="#0d0d0d"><meta name="description" content="一、常用命令首先列出Pod排查过程中我这边的常用命令：  查看Pod状态：kubectl get pod  podname -o wide 查看Pod的yaml配置：kubectl get pods podname -o yaml 查看pod事件：kubectl describe pods podname  查看容器日志：kubectl logs podsname -c  container-na">
<meta property="og:type" content="article">
<meta property="og:title" content="Pod异常状态排错">
<meta property="og:url" content="http://dycloudnavite.love/2022/08/15/Pod%E5%BC%82%E5%B8%B8%E7%8A%B6%E6%80%81%E6%8E%92%E9%94%99/index.html">
<meta property="og:site_name" content="J.のblog">
<meta property="og:description" content="一、常用命令首先列出Pod排查过程中我这边的常用命令：  查看Pod状态：kubectl get pod  podname -o wide 查看Pod的yaml配置：kubectl get pods podname -o yaml 查看pod事件：kubectl describe pods podname  查看容器日志：kubectl logs podsname -c  container-na">
<meta property="og:locale" content="zh_CN">
<meta property="og:image" content="http://dycloudnavite.love/img/kubernetes.png">
<meta property="article:published_time" content="2022-08-15T06:32:00.000Z">
<meta property="article:modified_time" content="2022-09-26T03:41:47.225Z">
<meta property="article:author" content="J.">
<meta property="article:tag" content="kubernetes">
<meta property="article:tag" content="云原生">
<meta name="twitter:card" content="summary">
<meta name="twitter:image" content="http://dycloudnavite.love/img/kubernetes.png"><link rel="shortcut icon" href="/dujie5416/img/touxiang.png"><link rel="canonical" href="http://dycloudnavite.love/2022/08/15/Pod%E5%BC%82%E5%B8%B8%E7%8A%B6%E6%80%81%E6%8E%92%E9%94%99/"><link rel="preconnect" href="//cdn.jsdelivr.net"/><link rel="preconnect" href="//busuanzi.ibruce.info"/><link rel="stylesheet" href="/dujie5416/css/index.css"><link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/@fortawesome/fontawesome-free/css/all.min.css" media="print" onload="this.media='all'"><link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/@fancyapps/ui/dist/fancybox.min.css" media="print" onload="this.media='all'"><script>const GLOBAL_CONFIG = { 
  root: '/dujie5416/',
  algolia: {"appId":"9GWB2VV7GN","apiKey":"dac56ec5b0b1f45281251ee59db4d637","indexName":"blog","hits":{"per_page":6},"languages":{"input_placeholder":"搜索文章","hits_empty":"找不到您查询的内容：${query}","hits_stats":"找到 ${hits} 条结果，用时 ${time} 毫秒"}},
  localSearch: undefined,
  translate: {"defaultEncoding":2,"translateDelay":0,"msgToTraditionalChinese":"繁","msgToSimplifiedChinese":"簡"},
  noticeOutdate: undefined,
  highlight: {"plugin":"highlighjs","highlightCopy":true,"highlightLang":true,"highlightHeightLimit":false},
  copy: {
    success: '复制成功',
    error: '复制错误',
    noSupport: '浏览器不支持'
  },
  relativeDate: {
    homepage: false,
    post: false
  },
  runtime: '天',
  date_suffix: {
    just: '刚刚',
    min: '分钟前',
    hour: '小时前',
    day: '天前',
    month: '个月前'
  },
  copyright: undefined,
  lightbox: 'fancybox',
  Snackbar: undefined,
  source: {
    justifiedGallery: {
      js: 'https://cdn.jsdelivr.net/npm/flickr-justified-gallery/dist/fjGallery.min.js',
      css: 'https://cdn.jsdelivr.net/npm/flickr-justified-gallery/dist/fjGallery.min.css'
    }
  },
  isPhotoFigcaption: false,
  islazyload: false,
  isAnchor: false
}</script><script id="config-diff">var GLOBAL_CONFIG_SITE = {
  title: 'Pod异常状态排错',
  isPost: true,
  isHome: false,
  isHighlightShrink: false,
  isToc: true,
  postUpdate: '2022-09-26 11:41:47'
}</script><noscript><style type="text/css">
  #nav {
    opacity: 1
  }
  .justified-gallery img {
    opacity: 1
  }

  #recent-posts time,
  #post-meta time {
    display: inline !important
  }
</style></noscript><script>(win=>{
    win.saveToLocal = {
      set: function setWithExpiry(key, value, ttl) {
        if (ttl === 0) return
        const now = new Date()
        const expiryDay = ttl * 86400000
        const item = {
          value: value,
          expiry: now.getTime() + expiryDay,
        }
        localStorage.setItem(key, JSON.stringify(item))
      },

      get: function getWithExpiry(key) {
        const itemStr = localStorage.getItem(key)

        if (!itemStr) {
          return undefined
        }
        const item = JSON.parse(itemStr)
        const now = new Date()

        if (now.getTime() > item.expiry) {
          localStorage.removeItem(key)
          return undefined
        }
        return item.value
      }
    }
  
    win.getScript = url => new Promise((resolve, reject) => {
      const script = document.createElement('script')
      script.src = url
      script.async = true
      script.onerror = reject
      script.onload = script.onreadystatechange = function() {
        const loadState = this.readyState
        if (loadState && loadState !== 'loaded' && loadState !== 'complete') return
        script.onload = script.onreadystatechange = null
        resolve()
      }
      document.head.appendChild(script)
    })
  
      win.activateDarkMode = function () {
        document.documentElement.setAttribute('data-theme', 'dark')
        if (document.querySelector('meta[name="theme-color"]') !== null) {
          document.querySelector('meta[name="theme-color"]').setAttribute('content', '#0d0d0d')
        }
      }
      win.activateLightMode = function () {
        document.documentElement.setAttribute('data-theme', 'light')
        if (document.querySelector('meta[name="theme-color"]') !== null) {
          document.querySelector('meta[name="theme-color"]').setAttribute('content', '#ffffff')
        }
      }
      const t = saveToLocal.get('theme')
    
          if (t === 'dark') activateDarkMode()
          else if (t === 'light') activateLightMode()
        
      const asideStatus = saveToLocal.get('aside-status')
      if (asideStatus !== undefined) {
        if (asideStatus === 'hide') {
          document.documentElement.classList.add('hide-aside')
        } else {
          document.documentElement.classList.remove('hide-aside')
        }
      }
    
    const detectApple = () => {
      if(/iPad|iPhone|iPod|Macintosh/.test(navigator.userAgent)){
        document.documentElement.classList.add('apple')
      }
    }
    detectApple()
    })(window)</script><link rel="stylesheet" href="/xxx.css"><link rel="stylesheet" href="/css/universe.css"><link rel="stylesheet" href="/css/custom.css"><style type="text/css">#toggle-sidebar {bottom: 80px}</style><link rel="stylesheet" href="https://fastly.jsdelivr.net/gh/tzy13755126023/BLOG_SOURCE/css/function.min.css"><link rel="stylesheet" href="/css/rightMenu.css"><style type="text/css">#toggle-sidebar {left:100px}</style><!-- hexo injector head_end start --><link rel="stylesheet" href="https://cdn.cbd.int/hexo-butterfly-clock-anzhiyu/lib/clock.min.css" /><!-- hexo injector head_end end --><meta name="generator" content="Hexo 6.2.0"><link rel="alternate" href="/dujie5416/atom.xml" title="J.のblog" type="application/atom+xml">
</head><body><div id="loading-box"><div class="loading-left-bg"></div><div class="loading-right-bg"></div><div class="spinner-box"><div class="configure-border-1"><div class="configure-core"></div></div><div class="configure-border-2"><div class="configure-core"></div></div><div class="loading-word">加载中...</div></div></div><div id="sidebar"><div id="menu-mask"></div><div id="sidebar-menus"><div class="avatar-img is-center"><img src="/dujie5416/img/touxiang.png" onerror="onerror=null;src='/img/friend_404.gif'" alt="avatar"/></div><div class="sidebar-site-data site-data is-center"><a href="/dujie5416/archives/"><div class="headline">文章</div><div class="length-num">73</div></a><a href="/dujie5416/tags/"><div class="headline">标签</div><div class="length-num">17</div></a><a href="/dujie5416/categories/"><div class="headline">分类</div><div class="length-num">11</div></a></div><hr/><div class="menus_items"><div class="menus_item"><a class="site-page" href="/dujie5416/"><i class="fa-fw fas fa-home"></i><span> 首页</span></a></div><div class="menus_item"><a class="site-page" href="/dujie5416/archives/"><i class="fa-fw fas fa-archive"></i><span> 归档</span></a></div><div class="menus_item"><a class="site-page" href="/dujie5416/tags/"><i class="fa-fw fas fa-tags"></i><span> 标签</span></a></div><div class="menus_item"><a class="site-page" href="/dujie5416/categories/"><i class="fa-fw fas fa-folder-open"></i><span> 分类</span></a></div></div></div></div><div class="post" id="body-wrap"><header class="post-bg" id="page-header" style="background-image: url('/dujie5416/img/longzhu.png')"><nav id="nav"><span id="blog_name"><a id="site-name" href="/dujie5416/">J.のblog</a></span><div id="menus"><div id="search-button"><a class="site-page social-icon search"><i class="fas fa-search fa-fw"></i><span> 搜索</span></a></div><div class="menus_items"><div class="menus_item"><a class="site-page" href="/dujie5416/"><i class="fa-fw fas fa-home"></i><span> 首页</span></a></div><div class="menus_item"><a class="site-page" href="/dujie5416/archives/"><i class="fa-fw fas fa-archive"></i><span> 归档</span></a></div><div class="menus_item"><a class="site-page" href="/dujie5416/tags/"><i class="fa-fw fas fa-tags"></i><span> 标签</span></a></div><div class="menus_item"><a class="site-page" href="/dujie5416/categories/"><i class="fa-fw fas fa-folder-open"></i><span> 分类</span></a></div></div><div id="toggle-menu"><a class="site-page"><i class="fas fa-bars fa-fw"></i></a></div></div></nav><div id="post-info"><h1 class="post-title">Pod异常状态排错</h1><div id="post-meta"><div class="meta-firstline"><span class="post-meta-date"><i class="far fa-calendar-alt fa-fw post-meta-icon"></i><span class="post-meta-label">发表于</span><time class="post-meta-date-created" datetime="2022-08-15T06:32:00.000Z" title="发表于 2022-08-15 14:32:00">2022-08-15</time><span class="post-meta-separator">|</span><i class="fas fa-history fa-fw post-meta-icon"></i><span class="post-meta-label">更新于</span><time class="post-meta-date-updated" datetime="2022-09-26T03:41:47.225Z" title="更新于 2022-09-26 11:41:47">2022-09-26</time></span><span class="post-meta-categories"><span class="post-meta-separator">|</span><i class="fas fa-inbox fa-fw post-meta-icon"></i><a class="post-meta-categories" href="/dujie5416/categories/kubernetes/">kubernetes</a></span></div><div class="meta-secondline"><span class="post-meta-separator">|</span><span class="post-meta-pv-cv" id="" data-flag-title="Pod异常状态排错"><i class="far fa-eye fa-fw post-meta-icon"></i><span class="post-meta-label">阅读量:</span><span id="busuanzi_value_page_pv"><i class="fa-solid fa-spinner fa-spin"></i></span></span></div></div></div></header><main class="layout" id="content-inner"><div id="post"><article class="post-content" id="article-container"><h1 id="一、常用命令"><a href="#一、常用命令" class="headerlink" title="一、常用命令"></a>一、常用命令</h1><p>首先列出Pod排查过程中我这边的常用命令：</p>
<ul>
<li>查看Pod状态：kubectl get pod  podname -o wide</li>
<li>查看Pod的yaml配置：kubectl get pods podname -o yaml</li>
<li>查看pod事件：kubectl describe pods podname </li>
<li>查看容器日志：kubectl logs podsname -c  container-name</li>
</ul>
<h1 id="二、Pod状态"><a href="#二、Pod状态" class="headerlink" title="二、Pod状态"></a>二、Pod状态</h1><ul>
<li>Error：Pod 启动过程中发生错误</li>
<li>NodeLost : Pod 所在节点失联</li>
<li>Unkown : Pod 所在节点失联或其它未知异常</li>
<li>Waiting : Pod 等待启动</li>
<li>Pending : Pod 等待被调度</li>
<li>ContainerCreating : Pod 容器正在被创建</li>
<li>Terminating : Pod 正在被销毁</li>
<li>CrashLoopBackOff ： 容器退出，kubelet 正在将它重启</li>
<li>InvalidImageName ： 无法解析镜像名称</li>
<li>ImageInspectError ： 无法校验镜像</li>
<li>ErrImageNeverPull ： 策略禁止拉取镜像</li>
<li>ImagePullBackOff ： 正在重试拉取</li>
<li>RegistryUnavailable ： 连接不到镜像中心</li>
<li>ErrImagePull ： 通用的拉取镜像出错</li>
<li>CreateContainerConfigError ： 不能创建 kubelet 使用的容器配置</li>
<li>CreateContainerError ： 创建容器失败</li>
<li>RunContainerError ： 启动容器失败</li>
<li>PreStartHookError : 执行 preStart hook 报错</li>
<li>PostStartHookError ： 执行 postStart hook 报错</li>
<li>ContainersNotInitialized ： 容器没有初始化完毕</li>
<li>ContainersNotReady ： 容器没有准备完毕</li>
<li>ContainerCreating ：容器创建中</li>
<li>PodInitializing ：pod 初始化中</li>
<li>DockerDaemonNotReady ：docker还没有完全启动</li>
<li>NetworkPluginNotReady ： 网络插件还没有完全启动</li>
</ul>
<h1 id="三、pod遇到的问题"><a href="#三、pod遇到的问题" class="headerlink" title="三、pod遇到的问题"></a>三、pod遇到的问题</h1><h2 id="1、pod一直处于Pending状态"><a href="#1、pod一直处于Pending状态" class="headerlink" title="1、pod一直处于Pending状态"></a>1、pod一直处于Pending状态</h2><p>Pending 状态说明 Pod 还没有被调度到某个节点上，需要看下 Pod 事件进一步判断原因，比如:</p>
<pre class="line-numbers language-none"><code class="language-none">$ kubectl describe pod tikv-0 
   . ... 
     Events: 
      Type     Reason   Age     From   Message 
       ----    ------   ----    ----    ------- 
      Warning FailedScheduling 3m (x106 over 33m) default-scheduler 0&#x2F;4 nodes are available: 1 node(s) had no available volume zone, 2 Insufficient cpu, 3 Insufficient memory.<span aria-hidden="true" class="line-numbers-rows"><span></span><span></span><span></span><span></span><span></span><span></span></span></code></pre>
<p>下面是我遇到的一些原因：</p>
<hr>
<ul>
<li><font color="#dd0000"><strong>节点资源不够</strong></font><br></li>
</ul>
<p>节点资源不够有以下几种情况:<br> 1、CPU负载过高<br> 2、剩余可被分配的内存不足<br> 3、剩余可用GPU数量不足<br> 如果判断某个 Node 资源是否足够？ 通过下面的命令查看node资源情况，关注以下信息：</p>
<blockquote>
<p>kubectl describe node nodename</p>
</blockquote>
<ul>
<li><code>Allocatable</code> : 表示此节点能够申请的资源总和</li>
<li><code>Allocated resources</code> : 表示此节点已分配的资源 (Allocatable 减去节点上所有 Pod 总 的 Request)</li>
</ul>
<p>前者与后者相减，可得出剩余可申请的资源。如果这个值小于 Pod 的 request，就不满足 Pod 的 资源要求，Scheduler 在 Predicates (预选) 阶段就会剔除掉这个 Node，也就不会调度上去。</p>
<hr>
<ul>
<li><font color="#dd0000"><strong>不满足 nodeSelector 与 affinity</strong></font><br></li>
</ul>
<blockquote>
<p>如果 Pod 包含 nodeSelector 指定了节点需要包含的 label，调度器将只会考虑将 Pod 调度到 包含这些 label 的Node 上，如果没有 Node 有这些 label 或者有这些 label 的 Node 其它 条件不满足也将会无法调度。参考官方文档</p>
</blockquote>
<p><a target="_blank" rel="noopener" href="https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#nodeselector">https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#nodeselector</a><br>如果 Pod 包含 affinity（亲和性）的配置，调度器根据调度算法也可能算出没有满足条件的 Node，从而无法调度。affinity 有以下几类:</p>
<ul>
<li><code>nodeAffinity</code>: 节点亲和性，可以看成是增强版的 nodeSelector，用于限制 Pod 只允许 被调度到某一部分 Node。</li>
<li><code>podAffinity</code>: Pod 亲和性，用于将一些有关联的 Pod 调度到同一个地方，同一个地方可以 是指同一个节点或同一个可用区的节点等。</li>
<li><code>podAntiAffinity</code>: Pod 反亲和性，用于避免将某一类 Pod 调度到同一个地方避免单点故 障，比如将集群 DNS 服务的 Pod 副本都调度到不同节点，避免一个节点挂了造成整个集群 DNS 解析失败，使得业务中断。</li>
</ul>
<hr>
<ul>
<li><font color="#dd0000"><strong>Node 存在 Pod 没有容忍的污点</strong></font><br></li>
</ul>
<blockquote>
<p>如果节点上存在污点 (Taints)，而 Pod 没有响应的容忍 (<code>Tolerations</code>)，Pod 也将不会调度上 去。通过 <code>describe node</code> 可以看下 Node 有哪些 Taints:</p>
</blockquote>
<pre class="line-numbers language-none"><code class="language-none">$ kubectl describe nodes host1 
    ... 
    Taints: special&#x3D;true:NoSchedule
    ...<span aria-hidden="true" class="line-numbers-rows"><span></span><span></span><span></span><span></span></span></code></pre>
<p>污点既可以是手动添加也可以是被自动添加，下面可以看一下。</p>
<p><strong>手动添加的污点：</strong><br>通过类似以下方式可以给节点添加污点:</p>
<pre class="line-numbers language-none"><code class="language-none">$ kubectl taint node host1 special&#x3D;true:NoSchedule 
  node &quot;host1&quot; tainted<span aria-hidden="true" class="line-numbers-rows"><span></span><span></span></span></code></pre>
<p>另外，有些场景下希望新加的节点默认不调度 Pod，直到调整完节点上某些配置才允许调度，就给新加 的节点都加上<code>node.kubernetes.io/unschedulable</code>   这个污点。</p>
<p><strong>自动添加的污点</strong><br>如果节点运行状态不正常，污点也可以被自动添加，从 v1.12 开始， TaintNodesByCondition 特性进入 Beta 默认开启，<code>controller manager</code> 会检查 Node 的 <code>Condition</code>，如果命中条件 就自动为 Node 加上相应的污点，这些 Condition 与 Taints 的对应关系如下:</p>
<pre class="line-numbers language-none"><code class="language-none">Conditon Value Taints 
-------- ----- ------
OutOfDisk True node.kubernetes.io&#x2F;out-of-disk
Ready False node.kubernetes.io&#x2F;not-ready
Ready Unknown node.kubernetes.io&#x2F;unreachable
MemoryPressure True node.kubernetes.io&#x2F;memory-pressure
PIDPressure True node.kubernetes.io&#x2F;pid-pressure
DiskPressure True node.kubernetes.io&#x2F;disk-pressure
NetworkUnavailable True node.kubernetes.io&#x2F;network-unavailable<span aria-hidden="true" class="line-numbers-rows"><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span><span></span></span></code></pre>
<p>解释下上面各种条件的意思:</p>
<ul>
<li>OutOfDisk 为 True 表示节点磁盘空间不够了 </li>
<li>Ready 为 False 表示节点不健康 </li>
<li>Ready 为 Unknown 表示节点失联，在 node-monitor-grace-period 这么长的时间内没有 上报状态 controller-manager 就会将 Node 状态置为 Unknown (默认 40s) </li>
<li>MemoryPressure 为 True 表示节点内存压力大，实际可用内存很少</li>
<li>PIDPressure 为 True 表示节点上运行了太多进程，PID 数量不够用了 </li>
<li>DiskPressure 为 True 表示节点上的磁盘可用空间太少了 </li>
<li>NetworkUnavailable 为 True 表示节点上的网络没有正确配置，无法跟其它 Pod 正常通 信</li>
</ul>
<p>另外，在云环境下，比如腾讯云 TKE，添加新节点会先给这个 Node 加上<br><code>node.cloudprovider.kubernetes.io/uninitialized</code>  的污点，等 Node 初始化成功后才自动移 除这个污点，避免 Pod 被调度到没初始化好的 Node 上。</p>
<hr>
<ul>
<li><font color="#dd0000"><strong>低版本 kube-scheduler 的 bug</strong></font><br></li>
</ul>
<p>可能是低版本 kube-scheduler 的 bug, 可以升级下调度器版本。</p>
<hr>
<ul>
<li><font color="#dd0000"><strong>kube-scheduler 没有正常运行</strong></font><br></li>
</ul>
<p>检查 maser 上的 kube-scheduler 是否运行正常，异常的话可以尝试重启临时恢复。</p>
<hr>
<ul>
<li><font color="#dd0000"><strong>驱逐后其它可用节点与当前节点有状态应用不在同一个可用区</strong></font><br></li>
</ul>
<blockquote>
<p>有时候服务部署成功运行过，但在某个时候节点突然挂了，此时就会触发驱逐，创建新的副本调度到其 它节点上，对于已经挂载了磁盘的 Pod，它通常需要被调度到跟当前节点和磁盘在同一个可用区，如果 集群中同一个可用区的节点不满足调度条件，即使其它可用区节点各种条件都满足，但不跟当前节点在 同一个可用区，也是不会调度的。为什么需要限制挂载了磁盘的 Pod 不能漂移到其它可用区的节点？ 试想一下，云上的磁盘虽然可以被动态挂载到不同机器，但也只是相对同一个数据中心，通常不允许跨数据中心挂载磁盘设备，因为网络时延会极大的降低 IO 速率。</p>
</blockquote>
<h2 id="2、Pod-一直处于-ContainerCreating-或-Waiting-状-态"><a href="#2、Pod-一直处于-ContainerCreating-或-Waiting-状-态" class="headerlink" title="2、Pod 一直处于 ContainerCreating 或 Waiting 状 态"></a>2、Pod 一直处于 ContainerCreating 或 Waiting 状 态</h2><hr>
<ul>
<li><p><font color="#dd0000"><strong>Pod 配置错误</strong></font><br></p>
</li>
<li><p>检查是否打包了正确的镜像</p>
</li>
<li><p>检查配置了正确的容器参数</p>
</li>
</ul>
<hr>
<ul>
<li><font color="#dd0000"><strong>挂载 Volume 失败</strong></font><br></li>
</ul>
<p>Volume 挂载失败也分许多种情况，先列下我这里目前已知的。</p>
<p><strong>Pod 漂移没有正常解挂之前的磁盘</strong></p>
<blockquote>
<p>在云尝试托管的 K8S 服务环境下，默认挂载的 Volume 一般是块存储类型的云硬盘，如果某个节点 挂了，kubelet 无法正常运行或与 apiserver 通信，到达时间阀值后会触发驱逐，自动在其它节点 上启动相同的副本 (Pod 漂移)，但是由于被驱逐的 Node 无法正常运行并不知道自己被驱逐了，也 就没有正常执行解挂，cloud-controller-manager 也在等解挂成功后再调用云厂商的接口将磁盘 真正从节点上解挂，通常会等到一个时间阀值后 cloud-controller-manager 会强制解挂云盘， 然后再将其挂载到 Pod 最新所在节点上，这种情况下 ContainerCreating 的时间相对长一点，但 一般最终是可以启动成功的，除非云厂商的 cloud-controller-manager 逻辑有 bug。</p>
</blockquote>
<hr>
<ul>
<li><font color="#dd0000"><strong>磁盘爆满</strong></font><br></li>
</ul>
<p>启动 Pod 会调 CRI 接口创建容器，容器运行时创建容器时通常会在数据目录下为新建的容器创建一 些目录和文件，如果数据目录所在的磁盘空间满了就会创建失败并报错:</p>
<pre class="line-numbers language-none"><code class="language-none">Events：
    Type Reason Age From
Message
     ---- ------ ---- ----
 Warning FailedCreatePodSandBox 2m (x4307 over 16h) kubelet, 10.179.80.31 (combined from similar events): Failed create pod sandbox: rpc error: code &#x3D; Unknown desc &#x3D; failed to create a sandbox for pod &quot;apigateway-6dc48bf8b6-l8xrw&quot;: Error response from daemon: mkdir &#x2F;var&#x2F;lib&#x2F;docker&#x2F;aufs&#x2F;mnt&#x2F;1f09d6c1c9f24e8daaea5bf33a4230de7dbc758e3b22785e8ee21e3e3d921214 no space left on device<span aria-hidden="true" class="line-numbers-rows"><span></span><span></span><span></span><span></span><span></span></span></code></pre>
<p>处理方法: <a href="https://dycloudnavite.love/2022/08/19/kubernetes%E7%A3%81%E7%9B%98%E7%88%86%E6%BB%A1%E6%95%85%E9%9A%9C%E5%A4%84%E7%90%86/index.html?_sw-precache=ec6633b042c26b4b5c4c0310bc4037a3">https://dycloudnavite.love/2022/08/19/kubernetes%E7%A3%81%E7%9B%98%E7%88%86%E6%BB%A1%E6%95%85%E9%9A%9C%E5%A4%84%E7%90%86/index.html?_sw-precache&#x3D;ec6633b042c26b4b5c4c0310bc4037a3</a></p>
<hr>
<ul>
<li><font color="#dd0000"><strong>节点内存碎片化</strong></font><br></li>
</ul>
<p>如果节点上内存碎片化严重，缺少大页内存，会导致即使总的剩余内存较多，但还是会申请内存失败，处理方法请看：<a href="https://dycloudnavite.love/2022/08/13/%E9%98%B2%E6%AD%A2%E5%86%85%E5%AD%98%E7%A2%8E%E7%89%87%E5%8C%96/index.html?_sw-precache=16ffedc87d255f33e2b942a0edddcb74">https://dycloudnavite.love/2022/08/13/%E9%98%B2%E6%AD%A2%E5%86%85%E5%AD%98%E7%A2%8E%E7%89%87%E5%8C%96/index.html?_sw-precache&#x3D;16ffedc87d255f33e2b942a0edddcb74</a></p>
<hr>
<ul>
<li><font color="#dd0000"><strong>limit 设置太小或者单位不对</strong></font><br></li>
</ul>
<p>如果 limit 设置过小以至于不足以成功运行 Sandbox 也会造成这种状态，常见的是因为 memory limit 单位设置不对造成的 limit 过小，比如误将 memory 的 limit 单位像 request 一样设 置为小 <strong>m</strong> ，这个单位在 memory 不适用，会被 k8s 识别成 byte， 应该用 <strong>Mi</strong> 或<strong>M</strong></p>
<p><em>举个例子: 如果 memory limit 设为 1024m 表示限制 1.024 Byte，这么小的内存， pause 容器一起来就会被 cgroup-oom kill 掉，导致 pod 状态一直处于 ContainerCreating。</em><br>这种情况通常会报下面的 event:</p>
<pre class="line-numbers language-none"><code class="language-none">Pod sandbox changed, it will be killed and re-created。<span aria-hidden="true" class="line-numbers-rows"><span></span></span></code></pre>
<p>kubelet 报错:</p>
<pre class="line-numbers language-none"><code class="language-none">to start sandbox container for pod ... Error response from daemon: OCI runtime create failed: container_linux.go:348: starting container process caused &quot;process_linux.go:301: running exec setns process for init caused \&quot;signal: killed\&quot;&quot;: unknown<span aria-hidden="true" class="line-numbers-rows"><span></span></span></code></pre>
<hr>
<ul>
<li><font color="#dd0000"><strong>拉取镜像失败</strong></font><br></li>
</ul>
<p>镜像拉取失败也分很多情况，这里列举下:</p>
<ul>
<li>配置了错误的镜像</li>
<li>Kubelet 无法访问镜像仓库（比如默认 pause 镜像在 gcr.io 上，国内环境访问需要特殊处 理）</li>
<li>拉取私有镜像的 <code>imagePullSecret</code> 没有配置或配置有误</li>
<li>镜像太大，拉取超时（可以适当调整 kubelet 的 <code>—image-pull-progress-deadline</code> 和 <code>—runtime-request-timeout</code> 选项）</li>
</ul>
<hr>
<ul>
<li><font color="#dd0000"><strong>CNI 网络错误</strong></font><br></li>
</ul>
<p>如果发生 CNI 网络错误通常需要检查下网络插件的配置和运行状态，如果没有正确配置或正常运行通 常表现为:</p>
<ul>
<li>无法配置 Pod 网络</li>
<li>无法分配 Pod IP</li>
</ul>
<hr>
<ul>
<li><font color="#dd0000"><strong>controller-manager 异常</strong></font><br></li>
</ul>
<p>查看 master 上 <code>kube-controller-manager</code> 状态，异常的话尝试重启。</p>
<hr>
<ul>
<li><font color="#dd0000"><strong>安装 docker 没删干净旧版本</strong></font><br></li>
</ul>
<p>如果节点上本身有 docker 或者没删干净，然后又安装 docker，比如在 centos 上用 yum 安装:</p>
<pre class="line-numbers language-none"><code class="language-none">1. yum install -y docker<span aria-hidden="true" class="line-numbers-rows"><span></span></span></code></pre>
<p>这样可能会导致 dockerd 创建容器一直不成功，从而 Pod 状态一直 <code>ContainerCreating</code>，查看 event 报错:</p>
<pre class="line-numbers language-none"><code class="language-none">     Type     Reason        Age       From
Message
      ----    ------       ----       ----
      Warning FailedCreatePodSandBox 18m (x3583 over 83m) kubelet, 192.168.4.5 (combined from similar events): Failed create pod sandbox: rpc error: code &#x3D; Unknown desc &#x3D; failed to start sandbox container for pod &quot;nginx-7db9fccd9b-2j6dh&quot;: Error response from daemon: ttrpc: client shutting down: read unix @-&gt;@&#x2F;containerd- shim&#x2F;moby&#x2F;de2bfeefc999af42783115acca62745e6798981dff75f4148fae8c086668f667&#x2F;shim.sock
 read: connection reset by peer: unknown
 Normal SandboxChanged 3m12s (x4420 over 83m) kubelet, 192.168.4.5
 Pod sandbox changed, it will be killed and re-created<span aria-hidden="true" class="line-numbers-rows"><span></span><span></span><span></span><span></span><span></span><span></span><span></span></span></code></pre>
<p>可能是因为重复安装 docker 版本不一致导致一些组件之间不兼容，从而导致 dockerd 无法正常创 建容器。</p>
<hr>
<ul>
<li><font color="#dd0000"><strong>存在同名容器</strong></font><br></li>
</ul>
<p>如果节点上已有同名容器，创建 sandbox 就会失败，event:</p>
<pre class="line-numbers language-none"><code class="language-none">Warning FailedCreatePodSandBox 2m kubelet, 10.205.8.91 Failed create pod sandbox: rpc error: code &#x3D; Unknown desc &#x3D; failed to create a sandbox for pod &quot;lomp-ext-d8c8b8c46-4v8tl&quot;: operation timeout: context deadline exceeded
Warning FailedCreatePodSandBox 3s (x12 over 2m) kubelet, 10.205.8.91 Failed create pod sandbox: rpc error: code &#x3D; Unknown desc &#x3D; failed to create a sandbox for pod &quot;lomp-ext-d8c8b8c46-4v8tl&quot;: Error response from daemon: Conflict. The container name &quot;&#x2F;k8s_POD_lomp-ext-d8c8b8c46- 4v8tl_default_65046a06-f795-11e9-9bb6-b67fb7a70bad_0&quot; is already in use by container &quot;30aa3f5847e0ce89e9d411e76783ba14accba7eb7743e605a10a9a862a72c1e2&quot;. You have to remove (or rename) that container to be able to reuse that name.<span aria-hidden="true" class="line-numbers-rows"><span></span><span></span></span></code></pre>
<p>关于什么情况下会产生同名容器，这个有待研究。</p>
<h2 id="3、Pod-处于-CrashLoopBackOff-状态"><a href="#3、Pod-处于-CrashLoopBackOff-状态" class="headerlink" title="3、Pod 处于 CrashLoopBackOff 状态"></a>3、Pod 处于 CrashLoopBackOff 状态</h2><p>Pod 如果处于 <code>CrashLoopBackOff</code> 状态说明之前是启动了，只是又异常退出了，只要 Pod 的 <code>restartPolicy</code>不是 Never 就可能被重启拉起，此时 Pod 的 <code>RestartCounts</code> 通常是大于 0 的，可以先看下容器进程的退出状态码来缩小问题范围</p>
<hr>
<ul>
<li><font color="#dd0000"><strong>容器进程主动退出：</strong></font><br></li>
</ul>
<p>如果是容器进程主动退出，退出状态码一般在 0-128 之间，除了可能是业务程序 BUG，还有其它许 多可能原因</p>
<hr>
<ul>
<li><font color="#dd0000"><strong>系统OOM</strong></font><br></li>
</ul>
<p>如果发生系统 OOM，可以看到 Pod 中容器退出状态码是 137，表示被 <code>SIGKILL</code> 信号杀死，同时 内核会报错: <code>Out of memory: Kill process</code> … 。大概率是节点上部署了其它非 K8S 管理的进 程消耗了比较多的内存，或者 kubelet 的 <code>--kube-reserved</code> 和 <code>--system-reserved</code> 配的 比较小，没有预留足够的空间给其它非容器进程，节点上所有 Pod 的实际内存占用总量不会超过 <code>/sys/fs/cgroup/memory/kubepods</code> 这里 cgroup 的限制，这个限制等于 <code>capacity - &quot;kube- reserved&quot; - &quot;system-reserved&quot;</code> ，如果预留空间设置合理，节点上其它非容器进程（kubelet, dockerd, kube-proxy, sshd 等) 内存占用没有超过 kubelet 配置的预留空间是不会发生系统 OOM 的，可以根据实际需求做合理的调整。</p>
<hr>
<ul>
<li><font color="#dd0000"><strong>系统OOM</strong></font><br></li>
</ul>
<p>如果是 cgrou OOM 杀掉的进程，从 Pod 事件的下 Reason 可以看到是 OOMKilled ，说明 容器实际占用的内存超过 limit 了，同时内核日志会报: &#96;&#96;。 可以根据需求调整下 limit。</p>
<hr>
<ul>
<li><font color="#dd0000"><strong>节点内存碎片化</strong></font><br></li>
</ul>
<p>如果节点上内存碎片化严重，缺少大页内存，会导致即使总的剩余内存较多，但还是会申请内存失败，</p>
<hr>
<ul>
<li><font color="#dd0000"><strong>健康检查失败</strong></font><br></li>
</ul>
<h2 id="4、Pod-一直处于-Terminating-状态"><a href="#4、Pod-一直处于-Terminating-状态" class="headerlink" title="4、Pod 一直处于 Terminating 状态"></a>4、Pod 一直处于 Terminating 状态</h2><hr>
<ul>
<li><font color="#dd0000"><strong>磁盘爆满</strong></font><br></li>
</ul>
<p>如果 docker 的数据目录所在磁盘被写满，docker 无法正常运行，无法进行删除和创建操作，所以 kubelet 调用 docker 删除容器没反应，看 event 类似这样：</p>
<pre class="line-numbers language-none"><code class="language-none">Normal Killing 39s (x735 over 15h) kubelet, 10.179.80.31 Killing container with id docker:&#x2F;&#x2F;apigateway:Need to kill Pod<span aria-hidden="true" class="line-numbers-rows"><span></span></span></code></pre>

<hr>
<ul>
<li><font color="#dd0000"><strong>存在 “i” 文件属性</strong></font><br></li>
</ul>
<p>如果容器的镜像本身或者容器启动后写入的文件存在 “i” 文件属性，此文件就无法被修改删除，而删 除 Pod 时会清理容器目录，但里面包含有不可删除的文件，就一直删不了，Pod 状态也将一直保持 Terminating，kubelet 报错:</p>
<pre class="line-numbers language-none"><code class="language-none">Sep 27 14:37:21 VM_0_7_centos kubelet[14109]: E0927 14:37:21.922965 14109 remote_runtime RemoveContainer &quot;19d837c77a3c294052a99ff9347c520bc8acb7b8b9a9dc9fab281fc09df38257&quot; failed: rpc error: code &#x3D; Unknown desc &#x3D; failed to remove container &quot;19d837c77a3c294052a99ff9347c520bc8acb7b8b9a9dc9fab281fc09df38257&quot;: Error response 19d837c77a3c294052a99ff9347c520bc8acb7b8b9a9dc9fab281fc09df38257: driver &quot;overlay2&quot; filesystem: remove &#x2F;data&#x2F;docker&#x2F;overlay2&#x2F;b1aea29c590aa9abda79f7cf3976422073fb3652757f0391db88534027546868 operation not permitted
Sep 27 14:37:21 VM_0_7_centos kubelet[14109]: E0927 14:37:21.923027 14109 kuberuntime_gc to remove container &quot;19d837c77a3c294052a99ff9347c520bc8acb7b8b9a9dc9fab281fc09df38257&quot; Unknown desc &#x3D; failed to remove container &quot;19d837c77a3c294052a99ff9347c520bc8acb7b8b9a9dc9fab281fc09df38257&quot;: Error response 19d837c77a3c294052a99ff9347c520bc8acb7b8b9a9dc9fab281fc09df38257: driver &quot;overlay2&quot; filesystem: remove &#x2F;data&#x2F;docker&#x2F;overlay2&#x2F;b1aea29c590aa9abda79f7cf3976422073fb3652757f0391db88534027546868 operation not permitted
<span aria-hidden="true" class="line-numbers-rows"><span></span><span></span></span></code></pre>
<p>通过 <strong>man chattr</strong> 查看 “i” 文件属性描述:</p>
<pre class="line-numbers language-none"><code class="language-none">A file with the &#39;i&#39; attribute cannot be modified: it cannot be deleted or renamed, no link can be created to this file and no data can be written to the file. Only the superuser or a process possessing the CAP_LINUX_IMMUTABLE capability can set or clear this attribute.<span aria-hidden="true" class="line-numbers-rows"><span></span></span></code></pre>
<p>彻底解决当然是不要在容器镜像中或启动后的容器设置 “i” 文件属性，临时恢复方法： 复制 kubelet 日志报错提示的文件路径，然后执行 chattr -i 文件名 :</p>
<pre class="line-numbers language-none"><code class="language-none">chattr -i &#x2F;data&#x2F;docker&#x2F;overlay2&#x2F;b1aea29c590aa9abda79f7cf3976422073fb3652757f0391db88534027<span aria-hidden="true" class="line-numbers-rows"><span></span></span></code></pre>
<p>执行完后等待 kubelet 自动重试，Pod 就可以被自动删除了。</p>
<hr>
<ul>
<li><font color="#dd0000"><strong>存在 Finalizers</strong></font><br></li>
</ul>
<p>k8s 资源的 metadata 里如果存在 <code>finalizers</code> ，那么该资源一般是由某程序创建的，并且在其 创建的资源的 metadata 里的 <code>finalizers</code> 加了一个它的标识，这意味着这个资源被删除时需要 由创建资源的程序来做删除前的清理，清理完了它需要将标识从该资源的 <code>finalizers</code> 中移除，然 后才会最终彻底删除资源。比如 Rancher 创建的一些资源就会写入 <code>finalizers</code> 标识。</p>
<p> 处理建议： kubectl edit 手动编辑资源定义，删掉 <code>finalizers</code> ，这时再看下资源，就会发现 已经删掉了</p>
<h2 id="5、Pod-一直处于-Unknown-状态"><a href="#5、Pod-一直处于-Unknown-状态" class="headerlink" title="5、Pod 一直处于 Unknown 状态"></a>5、Pod 一直处于 Unknown 状态</h2><p>通常是节点失联，没有上报状态给 apiserver，到达阀值后 controller-manager 认为节点失联 并将其状态置为 <code>Unknown</code><br>可能原因:</p>
<ul>
<li>节点高负载导致无法上报</li>
<li>节点宕机</li>
<li>节点被关机</li>
<li>网络不通</li>
</ul>
<h2 id="6、Pod-一直处于-Error-状态"><a href="#6、Pod-一直处于-Error-状态" class="headerlink" title="6、Pod 一直处于 Error 状态"></a>6、Pod 一直处于 Error 状态</h2><p>通常处于 Error 状态说明 Pod 启动过程中发生了错误。常见的原因包括：</p>
<ul>
<li>依赖的 ConfigMap、Secret 或者 PV 等不存在</li>
<li>请求的资源超过了管理员设置的限制，比如超过了 <code>LimitRange</code> 等</li>
<li>违反集群的安全策略，比如违反了 <code>PodSecurityPolicy</code> 等</li>
<li>容器无权操作集群内的资源，比如开启 RBAC 后，需要为 <code>ServiceAccount</code> 配置角色绑定</li>
</ul>
<h2 id="7、Pod-一直处于-ImagePullBackOff-状态"><a href="#7、Pod-一直处于-ImagePullBackOff-状态" class="headerlink" title="7、Pod 一直处于 ImagePullBackOff 状态"></a>7、Pod 一直处于 ImagePullBackOff 状态</h2><hr>
<ul>
<li><font color="#dd0000"><strong>http 类型 registry，地址未加入到 insecure- registry</strong></font><br></li>
</ul>
<p>dockerd 默认从 https 类型的 registry 拉取镜像，如果使用 https 类型的 registry，则 必须将它添加到 <code>insecure-registry</code> 参数中，然后重启或 reload dockerd 生效。</p>
<hr>
<ul>
<li><font color="#dd0000"><strong>https 自签发类型 resitry，没有给节点添加 ca 证书</strong></font><br></li>
</ul>
<p>如果 registry 是 https 类型，但证书是自签发的，dockerd 会校验 registry 的证书，校验 成功才能正常使用镜像仓库，要想校验成功就需要将 registry 的 ca 证书放置到<br><code>/etc/docker/certs.d/&lt;registry:port&gt;/ca.crt</code>  位置</p>
<hr>
<ul>
<li><font color="#dd0000"><strong>私有镜像仓库认证失败</strong></font><br></li>
</ul>
<p>如果 registry 需要认证，但是 Pod 没有配置 <code>imagePullSecret</code>，配置的 Secret 不存在或者 有误都会认证失败。</p>
<hr>
<ul>
<li><font color="#dd0000"><strong>镜像文件损坏</strong></font><br></li>
</ul>
<p>如果 push 的镜像文件损坏了，下载下来也用不了，需要重新 push 镜像文件。</p>
<hr>
<ul>
<li><font color="#dd0000"><strong>镜像拉取超时</strong></font><br></li>
</ul>
<p>如果节点上新起的 Pod 太多就会有许多可能会造成容器镜像下载排队，如果前面有许多大镜像需要下 载很长时间，后面排队的 Pod 就会报拉取超时。<br>kubelet 默认串行下载镜像:</p>
<pre class="line-numbers language-none"><code class="language-none">--serialize-image-pulls Pull images one at a time. We recommend *not* changing the default value on nodes that run docker daemon with version &lt; 1.9 or an Aufs storage backend. Issue #10959 has more details. (default true)<span aria-hidden="true" class="line-numbers-rows"><span></span></span></code></pre>
<p>也可以开启并行下载并控制并发:</p>
<pre class="line-numbers language-none"><code class="language-none">--registry-qps int32 If &gt; 0, limit registry pull QPS to this value. If 0, unlimited. (default 5)
--registry-burst int32 Maximum size of a bursty pulls, temporarily allows pulls to burst to this number, while still not exceeding registry-qps. Only used if --registry-qps &gt; 0 (default 10)<span aria-hidden="true" class="line-numbers-rows"><span></span><span></span></span></code></pre>

<hr>
<ul>
<li><font color="#dd0000"><strong>镜像不存在</strong></font><br></li>
</ul>
<p>kubelet日志：</p>
<pre class="line-numbers language-none"><code class="language-none">PullImage &quot;imroc&#x2F;test:v0.2&quot; from image service failed: rpc error: code &#x3D; Unknown desc &#x3D; Error response from daemon: manifest for imroc&#x2F;test:v0.2 not found<span aria-hidden="true" class="line-numbers-rows"><span></span></span></code></pre>
<h2 id="8、Pod-健康检查失败"><a href="#8、Pod-健康检查失败" class="headerlink" title="8、Pod 健康检查失败"></a>8、Pod 健康检查失败</h2><ul>
<li>Kubernetes 健康检查包含就绪检查(<code>readinessProbe</code>)和存活检查(<code>livenessProbe</code>)</li>
<li>pod 如果就绪检查失败会将此 pod ip 从 service 中摘除，通过 service 访问，流量将 不会被转发给就绪检查失败的 pod</li>
<li>pod 如果存活检查失败，kubelet 将会杀死容器并尝试重启</li>
</ul>
<p>健康检查失败的可能原因有多种，除了业务程序BUG导致不能响应健康检查导致 unhealthy，还能有 有其它原因，下面我们来逐个排查。</p>
<hr>
<ul>
<li><font color="#dd0000"><strong>健康检查配置不合理</strong></font><br></li>
</ul>
<p><code>initialDelaySeconds</code> 太短，容器启动慢，导致容器还没完全启动就开始探测，如果 <code>successThreshold</code> 是默认值 1，检查失败一次就会被 kill，然后 pod 一直这样被 kill 重 启。</p>
<hr>
<ul>
<li><font color="#dd0000"><strong>节点负载过高</strong></font><br></li>
</ul>
<p>cpu 占用高（比如跑满）会导致进程无法正常发包收包，通常会 timeout，导致 kubelet 认为 pod 不健康</p>
<hr>
<ul>
<li><font color="#dd0000"><strong>容器进程被木马进程杀死</strong></font><br></li>
</ul>
<hr>
<ul>
<li><font color="#dd0000"><strong>容器内进程端口监听挂掉</strong></font><br></li>
</ul>
<p>使用 <code>netstat -tunlp</code> 检查端口监听是否还在，如果不在了，抓包可以看到会直接 reset 掉健 康检查探测的连接:</p>
<pre class="line-numbers language-none"><code class="language-none">20:15:17.890996 IP 172.16.2.1.38074 &gt; 172.16.2.23.8888: Flags [S], seq 96880261, win 14600, options [mss 1424,nop,nop,sackOK,nop,wscale 7], length 0
20:15:17.891021 IP 172.16.2.23.8888 &gt; 172.16.2.1.38074: Flags [R.], seq 0, ack 96880262, win 0, length 0
20:15:17.906744 IP 10.0.0.16.54132 &gt; 172.16.2.23.8888: Flags [S], seq 1207014342, win 14600, options [mss 1424,nop,nop,sackOK,nop,wscale 7], length 0
20:15:17.906766 IP 172.16.2.23.8888 &gt; 10.0.0.16.54132: Flags [R.], seq 0, ack 1207014343, win 0, length 0<span aria-hidden="true" class="line-numbers-rows"><span></span><span></span><span></span><span></span></span></code></pre>
<p>连接异常，从而健康检查失败。发生这种情况的原因可能在一个节点上启动了多个使用 <code>hostNetwork</code> 监听相同宿主机端口的 Pod，只会有一个 Pod 监听成功，但监听失败的 Pod 的 业务逻辑允许了监听失败，并没有退出，Pod 又配了健康检查，kubelet 就会给 Pod 发送健康检查 探测报文，但 Pod 由于没有监听所以就会健康检查失败。</p>
<hr>
<ul>
<li><font color="#dd0000"><strong>SYN backlog 设置过小</strong></font><br></li>
</ul>
<p>SYN backlog 大小即 SYN 队列大小，如果短时间内新建连接比较多，而 SYN backlog 设置太 小，就会导致新建连接失败，通过 <code>netstat -s | grep TCPBacklogDrop</code>可以看到有多少是因为 backlog 满了导致丢弃的新连接。</p>
<p>如果确认是 backlog 满了导致的丢包，建议调高 backlog 的值，内核参数为 <code>net.ipv4.tcp_max_syn_backlog</code> 。</p>
<hr>
<ul>
<li><font color="#dd0000"><strong>容器进程主动退出</strong></font><br></li>
</ul>
<p>容器进程如果是自己主动退出(不是被外界中断杀死)，退出状态码一般在 0-128 之间，根据约定，正 常退出时状态码为 0，1-127 说明是程序发生异常，主动退出了，比如检测到启动的参数和条件不满 足要求，或者运行过程中发生 panic 但没有捕获处理导致程序退出。除了可能是业务程序 BUG，还 有其它许多可能原因，这里我们一一列举下。</p>
<ul>
<li><p><strong>DNS 无法解析</strong><br>可能程序依赖 集群 DNS 服务，比如启动时连接数据库，数据库使用 service 名称或外部域名都需 要 DNS 解析，如果解析失败程序将报错并主动退出。解析失败的可能原因:</p>
<p>1、集群网络有问题，Pod 连不上集群 DNS 服务<br>2、集群 DNS 服务挂了，无法响应解析请求<br>3、Service 或域名地址配置有误，本身是无法解析的地址</p>
</li>
<li><p><strong>程序配置有误</strong><br> 1、配置文件格式错误，程序启动解析配置失败报错退出<br> 2、配置内容不符合规范，比如配置中某个字段是必选但没有填写，配置校验不通过，程序报错主动 退出</p>
</li>
</ul>
</article><div class="post-copyright"><div class="post-copyright__author"><span class="post-copyright-meta">文章作者: </span><span class="post-copyright-info"><a href="http://dycloudnavite.love">J.</a></span></div><div class="post-copyright__type"><span class="post-copyright-meta">文章链接: </span><span class="post-copyright-info"><a href="http://dycloudnavite.love/2022/08/15/Pod%E5%BC%82%E5%B8%B8%E7%8A%B6%E6%80%81%E6%8E%92%E9%94%99/">http://dycloudnavite.love/2022/08/15/Pod%E5%BC%82%E5%B8%B8%E7%8A%B6%E6%80%81%E6%8E%92%E9%94%99/</a></span></div><div class="post-copyright__notice"><span class="post-copyright-meta">版权声明: </span><span class="post-copyright-info">本博客所有文章除特别声明外，均采用 <a href="https://creativecommons.org/licenses/by-nc-sa/4.0/" target="_blank">CC BY-NC-SA 4.0</a> 许可协议。转载请注明来自 <a href="http://dycloudnavite.love" target="_blank">J.のblog</a>！</span></div></div><div class="tag_share"><div class="post-meta__tag-list"><a class="post-meta__tags" href="/dujie5416/tags/kubernetes/">kubernetes</a><a class="post-meta__tags" href="/dujie5416/tags/%E4%BA%91%E5%8E%9F%E7%94%9F/">云原生</a></div><div class="post_share"><div class="addthis_inline_share_toolbox"></div><script src="//s7.addthis.com/js/300/addthis_widget.js#pubid=ra-5d034d84b42927bf" async="async"></script></div></div><div class="post-reward"><div class="reward-button"><i class="fas fa-qrcode"></i> 打赏</div><div class="reward-main"><ul class="reward-all"><li class="reward-item"><a href="/dujie5416/img/wechat.jpg" target="_blank"><img class="post-qr-code-img" src="/dujie5416/img/wechat.jpg" alt="微信"/></a><div class="post-qr-code-desc">微信</div></li><li class="reward-item"><a href="/dujie5416/img/alipay.jpg" target="_blank"><img class="post-qr-code-img" src="/dujie5416/img/alipay.jpg" alt="支付宝"/></a><div class="post-qr-code-desc">支付宝</div></li></ul></div></div><nav class="pagination-post" id="pagination"><div class="prev-post pull-left"><a href="/dujie5416/2022/08/16/Jenkins%E7%94%A8%E6%88%B7%E6%9D%83%E9%99%90%E6%8E%A7%E5%88%B6%E6%8F%92%E4%BB%B6%E2%80%94%E2%80%94Role-based%20Authorization%20Strategy/"><img class="prev-cover" src="/dujie5416/img/jenkins.png" onerror="onerror=null;src='/dujie5416/img/404.jpg'" alt="cover of previous post"><div class="pagination-info"><div class="label">上一篇</div><div class="prev_info">Jenkins用户权限控制插件——Role-based Authorization Strategy</div></div></a></div><div class="next-post pull-right"><a href="/dujie5416/2022/08/13/%E9%98%B2%E6%AD%A2%E5%86%85%E5%AD%98%E7%A2%8E%E7%89%87%E5%8C%96/"><img class="next-cover" src="/dujie5416/img/kubernetes.png" onerror="onerror=null;src='/dujie5416/img/404.jpg'" alt="cover of next post"><div class="pagination-info"><div class="label">下一篇</div><div class="next_info">防止内存碎片化</div></div></a></div></nav><div class="relatedPosts"><div class="headline"><i class="fas fa-thumbs-up fa-fw"></i><span>相关推荐</span></div><div class="relatedPosts-list"><div><a href="/dujie5416/2022/07/01/Deployment%E5%89%AF%E6%9C%AC%E6%8E%A7%E5%88%B6%E5%99%A8/" title="kubernetes常用资源——deployment"><img class="cover" src="/dujie5416/img/kubernetes.png" alt="cover"><div class="content is-center"><div class="date"><i class="far fa-calendar-alt fa-fw"></i> 2022-07-01</div><div class="title">kubernetes常用资源——deployment</div></div></a></div><div><a href="/dujie5416/2022/07/02/Configmap/" title="kubernetes常用资源——configMap"><img class="cover" src="/dujie5416/img/kubernetes.png" alt="cover"><div class="content is-center"><div class="date"><i class="far fa-calendar-alt fa-fw"></i> 2022-07-02</div><div class="title">kubernetes常用资源——configMap</div></div></a></div><div><a href="/dujie5416/2022/07/02/PV%20PVC/" title="kubernetes常用资源——PV&#x2F;PVC"><img class="cover" src="/dujie5416/img/kubernetes.png" alt="cover"><div class="content is-center"><div class="date"><i class="far fa-calendar-alt fa-fw"></i> 2022-07-02</div><div class="title">kubernetes常用资源——PV&#x2F;PVC</div></div></a></div><div><a href="/dujie5416/2022/07/01/RC%20RS%E5%89%AF%E6%9C%AC%E6%8E%A7%E5%88%B6%E5%99%A8/" title="kubernetes常用资源——RC&#x2F;RS"><img class="cover" src="/dujie5416/img/kubernetes.png" alt="cover"><div class="content is-center"><div class="date"><i class="far fa-calendar-alt fa-fw"></i> 2022-07-01</div><div class="title">kubernetes常用资源——RC&#x2F;RS</div></div></a></div><div><a href="/dujie5416/2022/07/03/Service%E6%8E%A7%E5%88%B6%E5%99%A8/" title="kubernetes常用资源——Service"><img class="cover" src="/dujie5416/img/kubernetes.png" alt="cover"><div class="content is-center"><div class="date"><i class="far fa-calendar-alt fa-fw"></i> 2022-07-03</div><div class="title">kubernetes常用资源——Service</div></div></a></div><div><a href="/dujie5416/2022/07/03/Volume%E5%AD%98%E5%82%A8%E5%8D%B7/" title="kubernetes常用资源——Volume存储卷"><img class="cover" src="/dujie5416/img/kubernetes.png" alt="cover"><div class="content is-center"><div class="date"><i class="far fa-calendar-alt fa-fw"></i> 2022-07-03</div><div class="title">kubernetes常用资源——Volume存储卷</div></div></a></div></div></div><hr/><div id="post-comment"><div class="comment-head"><div class="comment-headline"><i class="fas fa-comments fa-fw"></i><span> 评论</span></div></div><div class="comment-wrap"><div><div id="lv-container" data-id="city" data-uid="MTAyMC81Njk5OS8zMzQ2Mw=="></div></div></div></div></div><div class="aside-content" id="aside-content"><div class="card-widget card-info"><div class="is-center"><div class="avatar-img"><img src="/dujie5416/img/touxiang.png" onerror="this.onerror=null;this.src='/dujie5416/img/friend_404.gif'" alt="avatar"/></div><div class="author-info__name">J.</div><div class="author-info__description"></div></div><div class="card-info-data site-data is-center"><a href="/dujie5416/archives/"><div class="headline">文章</div><div class="length-num">73</div></a><a href="/dujie5416/tags/"><div class="headline">标签</div><div class="length-num">17</div></a><a href="/dujie5416/categories/"><div class="headline">分类</div><div class="length-num">11</div></a></div></div><div class="sticky_layout"><div class="card-widget" id="card-toc"><div class="item-headline"><i class="fas fa-stream"></i><span>目录</span><span class="toc-percentage"></span></div><div class="toc-content"><ol class="toc"><li class="toc-item toc-level-1"><a class="toc-link" href="#%E4%B8%80%E3%80%81%E5%B8%B8%E7%94%A8%E5%91%BD%E4%BB%A4"><span class="toc-number">1.</span> <span class="toc-text">一、常用命令</span></a></li><li class="toc-item toc-level-1"><a class="toc-link" href="#%E4%BA%8C%E3%80%81Pod%E7%8A%B6%E6%80%81"><span class="toc-number">2.</span> <span class="toc-text">二、Pod状态</span></a></li><li class="toc-item toc-level-1"><a class="toc-link" href="#%E4%B8%89%E3%80%81pod%E9%81%87%E5%88%B0%E7%9A%84%E9%97%AE%E9%A2%98"><span class="toc-number">3.</span> <span class="toc-text">三、pod遇到的问题</span></a><ol class="toc-child"><li class="toc-item toc-level-2"><a class="toc-link" href="#1%E3%80%81pod%E4%B8%80%E7%9B%B4%E5%A4%84%E4%BA%8EPending%E7%8A%B6%E6%80%81"><span class="toc-number">3.1.</span> <span class="toc-text">1、pod一直处于Pending状态</span></a></li><li class="toc-item toc-level-2"><a class="toc-link" href="#2%E3%80%81Pod-%E4%B8%80%E7%9B%B4%E5%A4%84%E4%BA%8E-ContainerCreating-%E6%88%96-Waiting-%E7%8A%B6-%E6%80%81"><span class="toc-number">3.2.</span> <span class="toc-text">2、Pod 一直处于 ContainerCreating 或 Waiting 状 态</span></a></li><li class="toc-item toc-level-2"><a class="toc-link" href="#3%E3%80%81Pod-%E5%A4%84%E4%BA%8E-CrashLoopBackOff-%E7%8A%B6%E6%80%81"><span class="toc-number">3.3.</span> <span class="toc-text">3、Pod 处于 CrashLoopBackOff 状态</span></a></li><li class="toc-item toc-level-2"><a class="toc-link" href="#4%E3%80%81Pod-%E4%B8%80%E7%9B%B4%E5%A4%84%E4%BA%8E-Terminating-%E7%8A%B6%E6%80%81"><span class="toc-number">3.4.</span> <span class="toc-text">4、Pod 一直处于 Terminating 状态</span></a></li><li class="toc-item toc-level-2"><a class="toc-link" href="#5%E3%80%81Pod-%E4%B8%80%E7%9B%B4%E5%A4%84%E4%BA%8E-Unknown-%E7%8A%B6%E6%80%81"><span class="toc-number">3.5.</span> <span class="toc-text">5、Pod 一直处于 Unknown 状态</span></a></li><li class="toc-item toc-level-2"><a class="toc-link" href="#6%E3%80%81Pod-%E4%B8%80%E7%9B%B4%E5%A4%84%E4%BA%8E-Error-%E7%8A%B6%E6%80%81"><span class="toc-number">3.6.</span> <span class="toc-text">6、Pod 一直处于 Error 状态</span></a></li><li class="toc-item toc-level-2"><a class="toc-link" href="#7%E3%80%81Pod-%E4%B8%80%E7%9B%B4%E5%A4%84%E4%BA%8E-ImagePullBackOff-%E7%8A%B6%E6%80%81"><span class="toc-number">3.7.</span> <span class="toc-text">7、Pod 一直处于 ImagePullBackOff 状态</span></a></li><li class="toc-item toc-level-2"><a class="toc-link" href="#8%E3%80%81Pod-%E5%81%A5%E5%BA%B7%E6%A3%80%E6%9F%A5%E5%A4%B1%E8%B4%A5"><span class="toc-number">3.8.</span> <span class="toc-text">8、Pod 健康检查失败</span></a></li></ol></li></ol></div></div><div class="card-widget card-recent-post"><div class="item-headline"><i class="fas fa-history"></i><span>最新文章</span></div><div class="aside-list"><div class="aside-list-item"><a class="thumbnail" href="/dujie5416/2023/07/29/%E6%A0%B9%E6%8D%AErouter%E6%98%BE%E7%A4%BA%E5%B7%A6%E4%BE%A7%E8%8F%9C%E5%8D%95%E6%A0%8F%E9%9A%90%E8%97%8F%E6%9F%90%E4%B8%AA%E8%8F%9C%E5%8D%95%E9%80%89%E9%A1%B9/" title="vue3左侧菜单隐藏某选项"><img src="/dujie5416/img/golang.png" onerror="this.onerror=null;this.src='/dujie5416/img/404.jpg'" alt="vue3左侧菜单隐藏某选项"/></a><div class="content"><a class="title" href="/dujie5416/2023/07/29/%E6%A0%B9%E6%8D%AErouter%E6%98%BE%E7%A4%BA%E5%B7%A6%E4%BE%A7%E8%8F%9C%E5%8D%95%E6%A0%8F%E9%9A%90%E8%97%8F%E6%9F%90%E4%B8%AA%E8%8F%9C%E5%8D%95%E9%80%89%E9%A1%B9/" title="vue3左侧菜单隐藏某选项">vue3左侧菜单隐藏某选项</a><time datetime="2023-07-29T06:32:00.000Z" title="发表于 2023-07-29 14:32:00">2023-07-29</time></div></div><div class="aside-list-item"><a class="thumbnail" href="/dujie5416/2023/07/13/Vue%E7%8A%B6%E6%80%81%E7%AE%A1%E7%90%86/" title="Vue状态管理"><img src="/dujie5416/img/golang.png" onerror="this.onerror=null;this.src='/dujie5416/img/404.jpg'" alt="Vue状态管理"/></a><div class="content"><a class="title" href="/dujie5416/2023/07/13/Vue%E7%8A%B6%E6%80%81%E7%AE%A1%E7%90%86/" title="Vue状态管理">Vue状态管理</a><time datetime="2023-07-13T09:29:00.000Z" title="发表于 2023-07-13 17:29:00">2023-07-13</time></div></div><div class="aside-list-item"><a class="thumbnail" href="/dujie5416/2023/06/27/%E5%9F%BA%E4%BA%8EIOC%E8%A7%A3%E5%86%B3%E5%AF%B9%E8%B1%A1%E4%BE%9D%E8%B5%96%E9%97%AE%E9%A2%98/" title="基于IOC解决对象依赖问题"><img src="/dujie5416/img/golang.png" onerror="this.onerror=null;this.src='/dujie5416/img/404.jpg'" alt="基于IOC解决对象依赖问题"/></a><div class="content"><a class="title" href="/dujie5416/2023/06/27/%E5%9F%BA%E4%BA%8EIOC%E8%A7%A3%E5%86%B3%E5%AF%B9%E8%B1%A1%E4%BE%9D%E8%B5%96%E9%97%AE%E9%A2%98/" title="基于IOC解决对象依赖问题">基于IOC解决对象依赖问题</a><time datetime="2023-06-27T03:32:00.000Z" title="发表于 2023-06-27 11:32:00">2023-06-27</time></div></div><div class="aside-list-item"><a class="thumbnail" href="/dujie5416/2023/06/23/Gin%E6%A1%86%E6%9E%B6/" title="Gin框架"><img src="/dujie5416/img/golang.png" onerror="this.onerror=null;this.src='/dujie5416/img/404.jpg'" alt="Gin框架"/></a><div class="content"><a class="title" href="/dujie5416/2023/06/23/Gin%E6%A1%86%E6%9E%B6/" title="Gin框架">Gin框架</a><time datetime="2023-06-23T06:32:00.000Z" title="发表于 2023-06-23 14:32:00">2023-06-23</time></div></div><div class="aside-list-item"><a class="thumbnail" href="/dujie5416/2023/06/04/ServeMux%E3%80%81httprouter/" title="ServeMux、httprouter"><img src="/dujie5416/img/golang.png" onerror="this.onerror=null;this.src='/dujie5416/img/404.jpg'" alt="ServeMux、httprouter"/></a><div class="content"><a class="title" href="/dujie5416/2023/06/04/ServeMux%E3%80%81httprouter/" title="ServeMux、httprouter">ServeMux、httprouter</a><time datetime="2023-06-04T03:32:00.000Z" title="发表于 2023-06-04 11:32:00">2023-06-04</time></div></div></div></div></div></div></main><footer id="footer" style="background-image: url('/dujie5416/img/longzhu.png')"><div id="footer-wrap"><div class="copyright">&copy;2019 - 2024  <i id="heartbeat" class="fa fas fa-heartbeat"></i> J.</div><div class="framework-info"><span>框架 </span><a target="_blank" rel="noopener" href="https://hexo.io">Hexo</a><span class="footer-separator">|</span><span>主题 </span><a target="_blank" rel="noopener" href="https://github.com/jerryc127/hexo-theme-butterfly">Butterfly</a><a href="http://beian.miit.gov.cn/"  style="color:#f72b07" target="_blank">京ICP备2022023567号</a></div></div><link rel="stylesheet" href="https://fastly.jsdelivr.net/gh/HCLonely/images@master/others/heartbeat.min.css"></footer></div><div id="rightside"><div id="rightside-config-hide"><button id="readmode" type="button" title="阅读模式"><i class="fas fa-book-open"></i></button><button id="translateLink" type="button" title="简繁转换">繁</button><button id="darkmode" type="button" title="浅色和深色模式转换"><i class="fas fa-adjust"></i></button><button id="hide-aside-btn" type="button" title="单栏和双栏切换"><i class="fas fa-arrows-alt-h"></i></button></div><div id="rightside-config-show"><button id="rightside_config" type="button" title="设置"><i class="fas fa-cog fa-spin"></i></button><button class="close" id="mobile-toc-button" type="button" title="目录"><i class="fas fa-list-ul"></i></button><a id="to_comment" href="#post-comment" title="直达评论"><i class="fas fa-comments"></i></a><button id="go-up" type="button" title="回到顶部"><i class="fas fa-arrow-up"></i></button></div></div><div id="algolia-search"><div class="search-dialog"><nav class="search-nav"><span class="search-dialog-title">搜索</span><button class="search-close-button"><i class="fas fa-times"></i></button></nav><div class="search-wrap"><div id="algolia-search-input"></div><hr/><div id="algolia-search-results"><div id="algolia-hits"></div><div id="algolia-pagination"></div><div id="algolia-info"><div class="algolia-stats"></div><div class="algolia-poweredBy"></div></div></div></div></div><div id="search-mask"></div></div><div id="rightMenu"><div class="rightMenu-group rightMenu-small"><div class="rightMenu-item" id="menu-backward"><i class="fa-solid fa-arrow-left"></i></div><div class="rightMenu-item" id="menu-forward"><i class="fa-solid fa-arrow-right"></i></div><div class="rightMenu-item" id="menu-refresh"><i class="fa-solid fa-arrow-rotate-right"></i></div><div class="rightMenu-item" id="menu-home"><i class="fa-solid fa-house"></i></div></div><div class="rightMenu-group rightMenu-line rightMenuOther"><a class="rightMenu-item menu-link" href="/archives/"><i class="fa-solid fa-archive"></i><span>文章归档</span></a><a class="rightMenu-item menu-link" href="/categories/"><i class="fa-solid fa-folder-open"></i><span>文章分类</span></a><a class="rightMenu-item menu-link" href="/tags/"><i class="fa-solid fa-tags"></i><span>文章标签</span></a></div><div class="rightMenu-group rightMenu-line rightMenuNormal"><a class="rightMenu-item menu-link" id="menu-radompage" href="/random/index.html"><i class="fa-solid fa-shoe-prints"></i><span>随便逛逛</span></a><div class="rightMenu-item" id="menu-translate"><i class="fa-solid fa-earth-asia"></i><span>繁简切换</span></div><div class="rightMenu-item" id="menu-darkmode"><i class="fa-solid fa-moon"></i><span>切换模式</span></div></div></div><div id="rightmenu-mask"></div><div><script src="/dujie5416/js/utils.js"></script><script src="/dujie5416/js/main.js"></script><script src="/dujie5416/js/tw_cn.js"></script><script src="https://cdn.jsdelivr.net/npm/@fancyapps/ui/dist/fancybox.umd.min.js"></script><script src="https://cdn.jsdelivr.net/npm/algoliasearch/dist/algoliasearch-lite.umd.min.js"></script><script src="https://cdn.jsdelivr.net/npm/instantsearch.js/dist/instantsearch.production.min.js"></script><script src="/dujie5416/js/search/algolia.js"></script><script>var preloader = {
  endLoading: () => {
    document.body.style.overflow = 'auto';
    document.getElementById('loading-box').classList.add("loaded")
  },
  initLoading: () => {
    document.body.style.overflow = '';
    document.getElementById('loading-box').classList.remove("loaded")

  }
}
window.addEventListener('load',preloader.endLoading())</script><div class="js-pjax"><script>function loadLivere () {
  if (typeof LivereTower === 'object') {
    window.LivereTower.init()
  }
  else {
    (function(d, s) {
        var j, e = d.getElementsByTagName(s)[0];
        if (typeof LivereTower === 'function') { return; }
        j = d.createElement(s);
        j.src = 'https://cdn-city.livere.com/js/embed.dist.js';
        j.async = true;
        e.parentNode.insertBefore(j, e);
    })(document, 'script');
  }
}

if ('Livere' === 'Livere' || !false) {
  if (false) btf.loadComment(document.getElementById('lv-container'), loadLivere)
  else loadLivere()
}
else {
  function loadOtherComment () {
    loadLivere()
  }
}</script></div><script defer src="https://npm.elemecdn.com/jquery@latest/dist/jquery.min.js"></script><script data-pjax defer src="https://npm.elemecdn.com/tzy-blog/lib/js/theme/chocolate.js"></script><canvas id="universe"></canvas><script defer src="/js/universe.js"></script><script defer data-pjax src="/js/rightMenu.js"></script><div class="aplayer no-destroy" data-id="7427714271" data-server="netease" data-type="playlist" data-fixed="true" data-mini="true" data-listFolded="false" data-order="random" data-lrctype="1" data-preload="none" data-autoplay="true" muted></div><script id="click-show-text" src="https://cdn.jsdelivr.net/npm/butterfly-extsrc/dist/click-show-text.min.js" data-mobile="false" data-text="I,LOVE,YOU" data-fontsize="15px" data-random="false" async="async"></script><link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/aplayer/dist/APlayer.min.css" media="print" onload="this.media='all'"><script src="https://cdn.jsdelivr.net/npm/aplayer/dist/APlayer.min.js"></script><script src="https://cdn.jsdelivr.net/npm/butterfly-extsrc/metingjs/dist/Meting.min.js"></script><script src="https://cdn.jsdelivr.net/npm/pjax/pjax.min.js"></script><script>let pjaxSelectors = ["meta[property=\"og:image\"]","meta[property=\"og:title\"]","meta[property=\"og:url\"]","head > title","#config-diff","#body-wrap","#rightside-config-hide","#rightside-config-show",".js-pjax"]

var pjax = new Pjax({
  elements: 'a:not([target="_blank"])',
  selectors: pjaxSelectors,
  cacheBust: false,
  analytics: false,
  scrollRestoration: false
})

document.addEventListener('pjax:send', function () {

  // removeEventListener scroll 
  window.tocScrollFn && window.removeEventListener('scroll', window.tocScrollFn)
  window.scrollCollect && window.removeEventListener('scroll', scrollCollect)

  typeof preloader === 'object' && preloader.initLoading()
  document.getElementById('rightside').style.cssText = "opacity: ''; transform: ''"
  
  if (window.aplayers) {
    for (let i = 0; i < window.aplayers.length; i++) {
      if (!window.aplayers[i].options.fixed) {
        window.aplayers[i].destroy()
      }
    }
  }

  typeof typed === 'object' && typed.destroy()

  //reset readmode
  const $bodyClassList = document.body.classList
  $bodyClassList.contains('read-mode') && $bodyClassList.remove('read-mode')

  typeof disqusjs === 'object' && disqusjs.destroy()
})

document.addEventListener('pjax:complete', function () {
  window.refreshFn()

  document.querySelectorAll('script[data-pjax]').forEach(item => {
    const newScript = document.createElement('script')
    const content = item.text || item.textContent || item.innerHTML || ""
    Array.from(item.attributes).forEach(attr => newScript.setAttribute(attr.name, attr.value))
    newScript.appendChild(document.createTextNode(content))
    item.parentNode.replaceChild(newScript, item)
  })

  GLOBAL_CONFIG.islazyload && window.lazyLoadInstance.update()

  typeof chatBtnFn === 'function' && chatBtnFn()
  typeof panguInit === 'function' && panguInit()

  // google analytics
  typeof gtag === 'function' && gtag('config', '', {'page_path': window.location.pathname});

  // baidu analytics
  typeof _hmt === 'object' && _hmt.push(['_trackPageview',window.location.pathname]);

  typeof loadMeting === 'function' && document.getElementsByClassName('aplayer').length && loadMeting()

  // prismjs
  typeof Prism === 'object' && Prism.highlightAll()

  typeof preloader === 'object' && preloader.endLoading()
})

document.addEventListener('pjax:error', (e) => {
  if (e.request.status === 404) {
    pjax.loadUrl('/404.html')
  }
})</script><script async data-pjax src="//busuanzi.ibruce.info/busuanzi/2.3/busuanzi.pure.mini.js"></script></div><!-- hexo injector body_end start --><script data-pjax>
  function butterfly_clock_anzhiyu_injector_config(){
    var parent_div_git = document.getElementsByClassName('sticky_layout')[0];
    var item_html = '<div class="card-widget card-clock"><div class="card-glass"><div class="card-background"><div class="card-content"><div id="hexo_electric_clock"><img class="entered loading" id="card-clock-loading" src="https://cdn.cbd.int/hexo-butterfly-clock-anzhiyu/lib/loading.gif" style="height: 120px; width: 100%;" data-ll-status="loading"/></div></div></div></div></div>';
    console.log('已挂载butterfly_clock_anzhiyu')
    if(parent_div_git) {
      parent_div_git.insertAdjacentHTML("afterbegin",item_html)
    }
  }
  var elist = 'null'.split(',');
  var cpage = location.pathname;
  var epage = 'all';
  var qweather_key = '56289903f0d448dfbe8262dba0175f70';
  var gaud_map_key = '01603f2b2d08d0e947bb101e9d9b148b';
  var baidu_ak_key = 'undefined';
  var flag = 0;
  var clock_rectangle = '112.982279,28.19409';
  var clock_default_rectangle_enable = 'false';

  for (var i=0;i<elist.length;i++){
    if (cpage.includes(elist[i])){
      flag++;
    }
  }

  if ((epage ==='all')&&(flag == 0)){
    butterfly_clock_anzhiyu_injector_config();
  }
  else if (epage === cpage){
    butterfly_clock_anzhiyu_injector_config();
  }
  </script><script src="https://widget.qweather.net/simple/static/js/he-simple-common.js?v=2.0"></script><script data-pjax src="https://cdn.cbd.int/hexo-butterfly-clock-anzhiyu/lib/clock.min.js"></script><!-- hexo injector body_end end --><script src="/dujie5416/live2dw/lib/L2Dwidget.min.js?094cbace49a39548bed64abff5988b05"></script><script>L2Dwidget.init({"tagMode":false,"debug":false,"model":{"jsonPath":"/dujie5416/live2dw/assets/tororo.model.json"},"display":{"position":"right","width":150,"height":350,"hOffset":20,"vOffset":-20},"mobile":{"show":false},"log":false,"pluginJsPath":"lib/","pluginModelPath":"assets/","pluginRootPath":"live2dw/"});</script><script async>window.onload=function(){var a=document.createElement('script'),b=document.getElementsByTagName('script')[0];a.type='text/javascript',a.async=!0,a.src='/sw-register.js?v='+Date.now(),b.parentNode.insertBefore(a,b)};</script></body></html>